import scrapy
import logging
from music_163_scrapy.items import SingerCategoryItem, SingerItem, SongItem, AlbumItem
import time
import re

class music_163(scrapy.Spider):
  name = 'music_163'
  allowed_domains = ['163.com']
  start_urls = ['http://music.163.com/discover/artist']

  def parse(self, response):
    # 歌手分类 如 华语男歌手，不含（推荐歌手，入驻歌手） 在其他组合中已包含
    items = SingerCategoryItem()
    singerCates = response.xpath(
            '//*[@id="singer-cat-nav"]/div/ul/li/a')
    for cIndex in range(len(singerCates)):
      # if cIndex == 0:
        cateText = singerCates[cIndex].xpath('text()').extract_first()
        cateHref = singerCates[cIndex].xpath('@href').extract_first()
        url = "http://music.163.com" + cateHref
        # print(cateText, cateHref)
        cateIDStr = re.sub(r'\D', '', cateHref)
        items['singerCateId'] = int(cateIDStr)
        items['singerCateName'] = cateText
        items['singerCateCreateAt'] = time.time()
        items['singerCateUrl'] = url
        yield items
        yield scrapy.http.Request(url, meta={
          'singerCateId': int(cateIDStr),
          'singerCateName': cateText
        }, callback=self.parse_singer_cate, dont_filter=True)

  def parse_singer_cate(self, response):
    # print('获取歌手分类A-Z链接')
    # 获取歌手分类A-Z链接
    # singerCateId = response.meta['singerCateId']
    # singerCateName = response.meta['singerCateName']
    # print(singerCateId, singerCateName)
    cateSorts = response.xpath('//*[@id="initial-selector"]/li/a')
    for sIndex in range(len(cateSorts)):
      if sIndex > 0 and sIndex < len(cateSorts) - 1:
      # if sIndex > 0 and sIndex < 2:
        sortText = cateSorts[sIndex].xpath('text()')[0].extract()
        sortHref = cateSorts[sIndex].xpath('@href')[0].extract()
        url = "http://music.163.com" + sortHref
        # print(sortText, sortHref)
        yield scrapy.http.Request(url, meta=response.meta, callback=self.parse_singer, dont_filter=True)

  def parse_singer(self, response):
    # 解析歌手
    # print('解析歌手')
    singerCateId = response.meta['singerCateId']
    singerCateName = response.meta['singerCateName']
    # print('-----------------parse_singer---------------------')
    # print(singerCateId, singerCateName)
    items = SingerItem()
    singerAddrs = response.xpath(
        '//*[@id="m-artist-box"]/li/a[contains(@href, "artist")]')
    for aIndex in range(len(singerAddrs)):
      # if aIndex < 1:
        addrText = singerAddrs[aIndex].xpath('text()')[0].extract()
        addrHref = singerAddrs[aIndex].xpath('@href')[0].extract()
        # print(addrText, addrHref)
        url = "http://music.163.com" + addrHref
        singerIDStr = re.sub(r'\D', '', addrHref)
        items['singerId'] = int(singerIDStr)
        items['singerName'] = addrText
        items['singerCreateAt'] = time.time()
        items['singerUrl'] = url
        items['singerCateId'] = singerCateId
        items['singerCateName'] = singerCateName
        yield items
        yield scrapy.http.Request(url, meta={
            'singerId': int(singerIDStr),
            'singerName': addrText
        }, callback=self.parse_music, dont_filter=True)

  def parse_music(self, response):
    # 解析50首热歌
    items = SongItem()
    singerId = response.meta['singerId']
    singerName = response.meta['singerName']
    musics = response.xpath('//*[@id="song-list-pre-cache"]//a[contains(@href, "song")]')
    albumHref = response.xpath('//*[@id="m_tabs"]/li[2]/a/@href').extract_first()
    albumUrl = 'http://music.163.com' + albumHref
    # print(albumUrl)
    for mIndex in range(len(musics)):
        songText = musics[mIndex].xpath('text()')[0].extract()
        songHref = musics[mIndex].xpath('@href')[0].extract()
        songIDStr = re.sub(r'\D', '', songHref)
        items['songId'] = int(songIDStr)
        items['songName'] = songText
        items['songCreateAt'] = time.time()
        items['songUrl'] = songHref
        items['singerIds'] = singerId
        items['singerNames'] = singerName
        yield items
    yield scrapy.http.Request(albumUrl, meta={
      'singerId': singerId,
      'singerName': singerName
    }, callback=self.parse_album, dont_filter=True)


  def parse_album (self, response):
    yield scrapy.http.Request(response.url, meta=response.meta, callback=self.parse_album_list, dont_filter=True)
    pages = response.css('.u-page a.zpgi[href*=album]::attr(href)').extract()
    for page in pages:
      url = 'http://music.163.com' + page
      yield scrapy.http.Request(url, meta=response.meta, callback=self.parse_album_list, dont_filter=True)
  
  def parse_album_list (self, response):
    singerId = response.meta['singerId']
    singerName = response.meta['singerName']
    albumList = response.xpath('//*[@id="m-song-module"]/li')
    items = AlbumItem()
    for aIndex in range(len(albumList)):
      albumName = albumList[aIndex].xpath('p[1]/a/text()').extract_first()
      albumHref = albumList[aIndex].xpath('p[1]/a/@href').extract_first()
      albumIdStr = re.sub(r'\D', '', albumHref)
      albumId = int(albumIdStr)
      # print('albumID', albumId)
      url = 'http://music.163.com' + albumHref
      yield scrapy.http.Request(url, meta={
          'albumName': albumName,
          'albumId': albumId,
          'singerId': singerId,
          'singerName': singerName
      }, callback=self.parse_album_song, dont_filter=True)
      
      items['albumId'] = albumId
      items['albumName'] = albumName
      items['albumCreateAt'] = time.time()
      items['albumUrl'] = url
      yield items

  def parse_album_song (self, response):
    singerId = response.meta['singerId']
    singerName = response.meta['singerName']
    albumId = response.meta['albumId']
    albumName = response.meta['albumName']
    items = SongItem()
    songList = response.css('.f-hide li a')
    for sIndex in range(len(songList)):
      songHref = songList[sIndex].xpath('@href').extract_first()
      songName = songList[sIndex].xpath('text()').extract_first()
      songIDStr = re.sub(r'\D', '', songHref)
      items['songId'] = int(songIDStr)
      items['songName'] = str(songName)
      items['songCreateAt'] = time.time()
      items['songUrl'] = songHref
      items['singerIds'] = singerId
      items['singerNames'] = singerName
      items['albumName'] = albumName
      items['albumId'] = albumId
      yield items
      # print(songName)
